package com.choudou5.spider.util;

import cn.hutool.core.date.DateUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.choudou5.solr.util.CollUtil;
import com.choudou5.spider.config.FieldBean;
import com.choudou5.spider.config.RespNextReqParam;
import com.choudou5.spider.config.SpiderConfigUtil;
import com.choudou5.spider.config.TaskBean;
import com.choudou5.spider.consts.SpiderConsts;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.*;

/**
 * Desc：爬虫任务工具类
 * Author：choudou5
 * Date：2018/6/29
 */
public class SpiderTaskUtil {

    private static final Logger logger = LoggerFactory.getLogger(SpiderTaskUtil.class);

    private static Random RAND = new Random();
    private static Map<String, Integer> TASK_PAGE = new HashMap<>();

    public static String checkUrlConstParam(TaskBean taskBo, String keyword){
        String taskId = taskBo.getId(),  url = taskBo.getApi();
        if(SpiderConsts.ParamConstType.PAGE_NO.hasKey(url)){
            Integer pageNo = TASK_PAGE.get(taskId);
            if(pageNo == null)
                pageNo = 0;
            pageNo++;
            url = StringUtils.replace(url, "{PAGE_NO}", pageNo.toString());
            TASK_PAGE.put(taskId, pageNo);
        }
        if(SpiderConsts.ParamConstType.PAGE_START.hasKey(url)){
            Integer pageNo = TASK_PAGE.get(taskId);
            if(pageNo == null)
                pageNo = 0;
            pageNo++;
            url = StringUtils.replace(url, "{PAGE_START}", ((pageNo - 1) * taskBo.getPageSize()) + "");
            TASK_PAGE.put(taskId, pageNo);
        }
        if(SpiderConsts.ParamConstType.KEYWORD.hasKey(url)){
            url = StringUtils.replace(url, "{KEYWORD}", keyword);
        }
        if(SpiderConsts.ParamConstType.RAND.hasKey(url)){
            url = StringUtils.replace(url, "{RAND}", RAND.nextInt(9999999) + "");
        }
        if(SpiderConsts.ParamConstType.REPLACE.hasKey(url)){
            //eg: information_id={:bottom_id}&amp;flag=down
            url = getReplaceParam(url, null);
        }
        return url;
    }

    private static String getReplaceParam(String url, RespNextReqParam nextReqParam){
        if(nextReqParam != null){
            List<String> list = new ArrayList<>(5);
            //eg: information_id={:bottom_id}&amp;flag=down
            String[] arr = StringUtils.split(url, SpiderConsts.ParamConstType.REPLACE.getKey());
            for (int i = 1; i < arr.length; i++) {
                list.add(StringUtils.substring(arr[i], 0, StringUtils.indexOf(arr[i], "}")));
            }
        }

        return url;
    }


    /**
     * 解析 分页行数据列表
     * @param taskBean
     * @param responseBody
     * @return
     */
    public static List<Map<String, String>> parsePageRowDataList(TaskBean taskBean, String responseBody, String keyword) {
        List<Map<String, String>> pageRowDataList = new ArrayList<>();
        logger.info("parsePageRowDataList type:" + taskBean.getParseType());
        //解析 html
        if (SpiderConsts.PARSE_HTML.equalsIgnoreCase(taskBean.getParseType())) {
            if (ParseDataUtil.isJsonContent(responseBody)) {
                logger.error("http返回了json，不能按html解析");
                return null;
            }
            pageRowDataList = getHtmlPageRowDataList(responseBody, taskBean, keyword);
        }
        //解析 json
        else if (SpiderConsts.PARSE_JSON.equalsIgnoreCase(taskBean.getParseType())) {
            if (!ParseDataUtil.isJsonContent(responseBody)) {
                logger.error("http返回非json，不能按json解析");
                logger.info(responseBody);
                return null;
            }
            pageRowDataList = getJsonPageRowDataList(responseBody, taskBean);
        }
        return pageRowDataList;
    }


    /**
     * 获取 分页数据列表
     * @param htmlBody
     * @param taskBean
     * @return
     */
    public static List<Map<String, String>> getHtmlPageRowDataList(String htmlBody, TaskBean taskBean, String keyword) {
        List<Map<String, String>> pageRowDataList = new ArrayList<>();
        //定位主体位置
        Elements contentEles = ParseDataUtil.jsoupReturnElements(htmlBody, taskBean.getContentLocationRule());
        if (contentEles == null || contentEles.size() == 0) {
            String errorMessage = " 解析html分页失败，定位规则：" + taskBean.getContentLocationRule() + " 不能定位到主体分页内容.";
            logger.error(errorMessage);
            return null;
        }
        //提取字段
        List<FieldBean> fieldBeans = taskBean.getTable().getFields();
        Iterator<Element> iter = contentEles.iterator();
        while (iter.hasNext()) {
            Element contentEle = iter.next();
            Elements rows = contentEle.children();
            Map<String, String> fields = null;
            for (Element rowEle : rows) {
                fields = parseJsoupPageRow(rowEle, fieldBeans, keyword);
                if(MapUtil.isNotEmpty(fields)){
                    pageRowDataList.add(fields);
                }
            }
        }
        return pageRowDataList;
    }


    /**
     * 获取 分页数据列表
     * @param jsonBody
     * @param taskBean
     * @return
     */
    public static List<Map<String, String>> getJsonPageRowDataList(String jsonBody, TaskBean taskBean) {
        List<Map<String, String>> pageRowDataList = new ArrayList<>();
        JSONArray jsonArray = null;
        //定位主体位置
        if (StringUtils.isNotBlank(taskBean.getContentLocationRule())) {
            jsonArray = ParseDataUtil.jsonArray(jsonBody, taskBean.getContentLocationRule());
        } else {
            jsonArray = ParseDataUtil.jsonArray(jsonBody);
        }
        if (jsonArray == null || jsonArray.size() == 0) {
            String errorMessage = " 解析json分页失败，定位规则：" + taskBean.getContentLocationRule() + " 不能定位到主体分页内容.";
            logger.error(errorMessage);
            return null;
        }
        //提取字段
        List<FieldBean> fieldRuleBos = taskBean.getTable().getFields();
        int len = jsonArray.size();
        Map<String, String> fields = null;
        for (int i = 0; i < len; i++) {
            JSONObject jsonObject = jsonArray.getJSONObject(i);
            fields = parseJsonRow(jsonObject, fieldRuleBos);
            pageRowDataList.add(fields);
        }
        return pageRowDataList;
    }


    /**
     * 解析行
     *
     * @param rowEle
     * @param fieldBeans
     * @return
     */
    private static Map<String, String> parseJsoupPageRow(Element rowEle, List<FieldBean> fieldBeans, String keyword) {
        String rowHtml = rowEle.toString();
        Map<String, String> fields = new LinkedHashMap<>();
        String fieldVal = null;
        Set<String> words = new HashSet<>();
        for (FieldBean fieldBean : fieldBeans) {
            fieldVal = fieldBean.getDef();
            if("KEYWORD".equals(fieldVal)){
                fields.put(fieldBean.getName(), keyword);
                continue;
            }
            if(StringUtils.isNotBlank(fieldBean.getExpr())){
                //获取解析字段值
                fieldVal = getHtmlParseFieldValue(fieldBean, rowHtml);
                if (StringUtils.isBlank(fieldVal)) {
                    logger.warn("解析失败, {} 规则为[{}]", fieldBean.getName(), fieldBean.getExpr());
                }
            }
            //默认时间
            if("NOW".equals(fieldBean.getDef()) && (fieldVal == null || "NOW".equals(fieldVal)))
                fieldVal = DateUtil.now();
            //必填字段
            if("true".equals(fieldBean.getRequired()) && StrUtil.isBlank(fieldVal)) {
                fields.clear();
                break;
            }
            //提取关键字
            if(StrUtil.isNotBlank(fieldBean.getExtractWordTag())){
                Set<String> wordList = ParseDataUtil.jsoupExtractTagText(fieldVal, fieldBean.getExtractWordTag());
                if(CollUtil.isNotEmpty(wordList)){
                    words.addAll(wordList);
                }
                fieldVal = ParseDataUtil.jsoupFormatToText(fieldVal);
            }
            //移除内容
            if(StrUtil.isNotBlank(fieldBean.getRemoveCont())){
                fieldVal = StrUtil.replace(fieldVal, fieldBean.getRemoveCont(), "");
            }
            fields.put(com.choudou5.solr.util.StrUtil.toJavaVariableName(fieldBean.getName()), fieldVal);
        }
        if(!words.isEmpty()){
            fields.put("keywords", StrUtil.join("|", words));
        }
        return fields;
    }

    public static Map<String, String> getParseResult(String taskId, String url, String rowHtml) {
        List<FieldBean> fieldRuleList = SpiderConfigUtil.getTaskBean(taskId).getTable().getFields();
        Map<String, String> fields = new LinkedHashMap<>();
        String fieldVal;
        for (FieldBean fieldRuleBo : fieldRuleList) {
            //获取 解析值
            fieldVal = getHtmlParseFieldValue(fieldRuleBo, rowHtml);
            if (StringUtils.isBlank(fieldVal)) {
                String currentLog = String.format("字段[%s]解析失败,  规则为[%s]", fieldRuleBo.getDesc(),  fieldRuleBo.getExpr());
                logger.error(currentLog);
            } else {
                fields.put(fieldRuleBo.getName(), fieldVal);
            }
        }
        return fields;
    }


    /**
     * 获取 html 解析字段值
     * @param fieldRuleBo
     * @param rowHtml
     */
    private static String getHtmlParseFieldValue(FieldBean fieldRuleBo, String rowHtml) {
        String fieldVal = null;
        String ruleType = fieldRuleBo.getRule();
        if(ruleType==null)
            ruleType = SpiderConsts.RULE_JSOUP;
        String fieldExpr = fieldRuleBo.getExpr();
        String fieldName = fieldRuleBo.getName();
        String extractWordTag = fieldRuleBo.getExtractWordTag();
        try {
            switch (ruleType) {
                case SpiderConsts.RULE_JSOUP:
                    fieldVal = ParseDataUtil.jsoup(rowHtml, fieldExpr, StrUtil.isNotBlank(extractWordTag));
                    break;
                case SpiderConsts.RULE_REGEXP:
                    fieldVal = ParseDataUtil.firstMatchData(rowHtml, fieldExpr);
                    break;
                case SpiderConsts.RULE_SUBSTRING:
                    fieldVal = ParseDataUtil.substring(rowHtml, fieldExpr, fieldName);
                    break;
            }
        } catch (Exception e) {
            logger.error("解析“" + fieldName + "”失败", e);
        }
        return fieldVal;
    }

    /**
     * 解析 Json行
     * @param jsonObject
     * @param sqlFieldBeans
     * @return
     */
    private static Map<String, String> parseJsonRow(JSONObject jsonObject, List<FieldBean> sqlFieldBeans) {
        Map<String, String> fields = new LinkedHashMap<>();
        String fieldVal = null;
        for (FieldBean sqlFieldBean : sqlFieldBeans) {
            fieldVal = sqlFieldBean.getDef();
            if (StringUtils.isNotBlank(sqlFieldBean.getExpr())) {
                fieldVal = ParseDataUtil.jsonVal(jsonObject, sqlFieldBean.getExpr());
            }
            //处理转换值
            if(fieldVal != null && StrUtil.isNotBlank(sqlFieldBean.getConvert())){
                Map<String, String> convert = sqlFieldBean.getConvertMap();
                fieldVal = convert.get(fieldVal);
            }
            fields.put(sqlFieldBean.getName(), fieldVal);
        }
        return fields;
    }

    /**
     * 获取 随机休眠秒 (5秒内随机)
     * @return
     */
    public static int getRadSleepMs() {
        return RAND.nextInt(3000);
    }


}
